import emoji
import pandas as pd
from data_utils.preprocess import process
import json
import os
import re


def count1():
    """
    观察错误样例中有表情的
    """
    fold = 4
    path = r'./dev_fold{}.csv'.format(fold)
    df = pd.read_csv(path, encoding='utf-8')
    content = df['content']
    # content = df['processed']

    data = []
    true_label = []
    predict_label = []

    for c, true, predict in zip(content, df['true'], df['predict']):
        if predict != true:  # 错例
            if emoji.emoji_count(str(c)) > 0:  # 有表情
                # print(true, predict, c)
                data.append(c.strip())
                true_label.append(str(true))
                predict_label.append(str(predict))

    frame = pd.DataFrame()
    frame['content'] = data
    frame['true'] = true_label
    frame['predict'] = predict_label
    frame.to_csv(r'./emoji_fold{}.csv'.format(fold), index=0, encoding='utf-8')


def emoji_count(data):

    emoji_pic = {}  # 图片表情
    emoji_char = {}  # 文字表情
    emoji_sentences_count = 0  # 有表情的句子

    for content in data:
        have_emoji_char = False
        have_emoji_pic = False
        if emoji.emoji_count(content) > 0:
            have_emoji_pic = True
            for e in emoji.emoji_lis(content):
                if e['emoji'] in emoji_pic:
                    emoji_pic[e['emoji']] += 1
                else:
                    emoji_pic[e['emoji']] = 1
        match = re.findall(r'\[.*?\]', content)
        if match:
            have_emoji_char = True
            for i in match:
                if i in emoji_char:
                    emoji_char[i] += 1
                else:
                    emoji_char[i] = 1
        if have_emoji_char or have_emoji_pic:
            emoji_sentences_count += 1

    # 按照出现次数从高到低排序
    emoji_char = sorted(emoji_char.items(), key=lambda x: x[1], reverse=True)
    emoji_pic = sorted(emoji_pic.items(), key=lambda x: x[1], reverse=True)
    print('含表情句子数:{}'.format(emoji_sentences_count))
    print('覆盖率:{}%\t({}/{})'.format(round(100*emoji_sentences_count/len(data), 3), emoji_sentences_count, len(data)))
    print('文字表情{}种\t符号表情{}种'.format(len(emoji_char), len(emoji_pic)))
    print(emoji_char)
    print(emoji_pic)
    print()


def emoji_count2():
    paths = [
        r'../raw/train/usual_train.txt',
        r'../raw/train/virus_train.txt',
        r'../raw/eval/usual_eval.txt',
        r'../raw/eval/virus_eval.txt'
    ]
    # 合并统计
    data = []
    for p in paths:
        print(os.path.basename(p))
        with open(p, 'r', encoding='utf-8') as f:
            temp = json.load(f)
        data.extend([i['content'] for i in temp])

    emoji_dict = {}
    emoji_char = {}
    count = 0
    with open(r'../data/emoji_dict.json', 'r', encoding='utf-8') as f:
        emoji2zh = json.load(f)
    for content in data:
        content, _ = process([content], [0], drop_empty=False)
        content = content[0]
        if emoji.emoji_count(content) > 0:
            for e in emoji.emoji_lis(content):
                if e['emoji'] in emoji2zh:
                    key = '[{}]'.format(emoji2zh[e['emoji']])
                    if key in emoji_char:
                        emoji_char[key] += 1
                    else:
                        emoji_char[key] = 1
                else:
                    pass
        match = re.findall(r'\[.*?\]', content)
        if match:
            for i in match:
                if i in emoji_char:
                    emoji_char[i] += 1
                else:
                    emoji_char[i] = 1
    emoji_char = sorted(emoji_char.items(), key=lambda x: x[1], reverse=True)
    for i in emoji_char:
        print(i[0], end=' ')
    # print(count)
    # print(unseen)



if __name__ == '__main__':
    # paths = [
    #     r'../raw/train/usual_train.txt',
    #     r'../raw/train/virus_train.txt',
    #     r'../raw/eval/usual_eval.txt',
    #     r'../raw/eval/virus_eval.txt'
    # ]
    # 分开统计
    # for p in paths:
    #     print(os.path.basename(p))
    #     with open(p, 'r', encoding='utf-8') as f:
    #         temp = json.load(f)
    #     emoji_count([i['content'] for i in temp])

    # 合并统计
    # data = []
    # for p in paths:
    #     print(os.path.basename(p))
    #     with open(p, 'r', encoding='utf-8') as f:
    #         temp = json.load(f)
    #     data.extend([i['content'] for i in temp])

    # emoji_count(data)


    emoji_count2()
